In [1]:
import requests as r
import re
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import seaborn as sns
import time
import matplotlib.pyplot as plt
%matplotlib inline
sns.set()
import warnings
warnings.simplefilter('ignore') #ignore the warnings, not the errors
In [2]:
# happy to say everything will work right now, not like the last time
Why¶
everyone is the same - no need of machine learning to predict
Results¶
What makes me happy?¶
- money ?
- cars ?
- films ?
Data¶
Types of data that can influence my mood:
- money
- sport data
- activities I do
Sharing personal info:
- data is touchy - mood is limited by category
- IronHack is excluded
- money are not in EUR, smth like YEN
Mood¶
Mood App¶
 |
 |
 |
Mood Exploratory¶
Mood Wrangle¶
In [3]:
mood=pd.read_csv('python_data/Daylio_01.2019_07.2020_date.mood.mood_text.activities.csv')
In [4]:
#separate activities by dummies
mood=pd.concat([mood[['full_date','mood_num', 'mood']],mood.activities.str.replace(' ','').str.get_dummies(sep='|')], axis=1)
In [5]:
#leave only the columns I feel comfortable sharing
mood=mood[['full_date', 'mood_num', 'mood','4pda/search', 'biking(20m+)', 'cleaning',
'cook(nomicrow)', 'daysleep', 'drums(GHincluded)',
'eatingout', 'films', 'friends', 'gaming', 'italiano', 'laundry',
'maladie', 'party/bar', 'people(1h+)', 'personalwork',
'reading', 'shopping(big)', 'shopping(courses)', 'sport',
'stressventre', 'sun(onskin)', 'television', 'travel',
'volley', 'walking(30m+)', 'workplacework']]
#rename somecolumns for better readability
mood.rename(columns={'full_date': "date",
"mood_num": "mood_float",
'sun(onskin)':'sunnyday',
'cook(nomicrow)':'cook',
'biking(20m+)':'biking',
'drums(GHincluded)':'drums',
'walking(30m+)':'walking',
'4pda/search': 'repairing',
'volley': "volley_ball",
'shopping(courses)': "supermarket",
'people(1h+)': "people",
'maladie': "illness",
'shopping(big)':'shopping',
'stressventre':'stress'
}, inplace=True)
In [6]:
#date to datetime
mood.date=pd.to_datetime(mood.date)
#sort values and reset index
mood.sort_values(by='date',ascending=True,inplace=True)
mood=mood.reset_index(drop=True)
In [7]:
mood.head()
Out[7]:
Money¶
Money app¶
 |
 |
 |
Money Exploratory¶
Money Wrangle¶
In [8]:
money=pd.read_csv('python_data/Monefy_01.2019_07.2020_Bills_CarteResto_Voyage.csv')
money_sp=pd.read_csv('python_data/Monefy_01.2019_07.2020_NO_Bills_NO_Voyage_With_CarteResto.csv')
money_no_bills=pd.read_csv('python_data/Monefy_01.2019_07.2020_NO_Bills_With_Voyage_With_CarteResto.csv')
In [9]:
#date to datetime
money.date=pd.to_datetime(money.date)
money_sp.date=pd.to_datetime(money_sp.date)
money_no_bills.date=pd.to_datetime(money_no_bills.date)
In [10]:
money.head()
Out[10]:
Steps¶
Steps app¶
 |
 |
 |
Steps Exploratory¶
In [11]:
steps_lol=pd.read_csv('miband_data/ACTIVITY.csv')
steps_lol.date=pd.to_datetime(steps_lol.date)
steps_lol['year']=steps_lol.date.dt.year
steps_lol.year.value_counts()
Out[11]:
Wrangling¶
In [12]:
steps=pd.read_csv('miband_data/ACTIVITY_MINUTE.csv')
In [13]:
steps.date=pd.to_datetime(steps.date)
In [14]:
steps.head()
Out[14]:
Visualize¶
Mood¶
In [15]:
import plotly.express as px
fig = px.pie(mood, values='mood_float', names='mood', title='Pie chart for Eldiias',height=400, hole=.3)
fig.update_traces(textposition='inside', textinfo='percent+label')
#, hole=.3
fig.show()
 |
 |
Money¶
 |
 |
In [16]:
money.head()
Out[16]:
In [17]:
#separate to bills and everything else
money['Bills'] = np.where(money['category']=='Bills', 'Bills', 'Not_Bills')
import plotly.express as px
fig = px.pie(money, values='amount', names='Bills', title='My spendings',height=400)
fig.update_traces(textposition='inside', textinfo='percent+label')
#, hole=.3
fig.show()
Merge¶
In [18]:
#pivot table to sum all the buyings per day
money.pivot_table(index='date', values='amount', aggfunc='sum').head()
Out[18]:
In [19]:
# add that pivot table to mood dataset
mm=mood.join(money.pivot_table(index='date', values='amount', aggfunc='sum'), on='date')
In [20]:
# preparing table to analysis
mm['amount_spent']=mm.amount.fillna(0)
mm.drop(index=1, inplace=True) #drop a duplicate row
mm.drop('amount', axis=1,inplace=True)
mm.drop('mood', axis=1,inplace=True)
mm.reset_index(inplace=True, drop=True)
In [21]:
#pivot table to sum all the steps per day
steps.pivot_table(index='date', values='steps', aggfunc='sum')
Out[21]:
In [22]:
# select only needed dates
steps_pivot=steps.pivot_table(index='date', values='steps', aggfunc='sum')
steps_pivot=steps_pivot.query('date>="2019-01-01" & date<="2020-07-31"')
In [23]:
# add that pivot table to mood dataset
mm_st=mm.join(steps_pivot, on='date')
In [24]:
#fillnawith average
mm_st['steps'].fillna((mm_st['steps'].mean()), inplace=True)
In [25]:
mm_st.head()
Out[25]:
Analyze¶
Activity influence¶
In [26]:
mm
Out[26]:
In [27]:
# recognize if an activity adds something to my mood
mm.pivot_table(index='sport', values='mood_float', aggfunc='mean')
#show number of rows
Out[27]:
In [28]:
#let's get a table of that
#specify columns
cols=['repairing', 'biking', 'cleaning', 'cook',
'daysleep', 'drums', 'eatingout', 'films', 'friends', 'gaming',
'italiano', 'laundry', 'illness', 'party/bar', 'people', 'personalwork',
'reading', 'shopping', 'supermarket', 'sport', 'stress',
'sunnyday', 'television', 'travel', 'volley_ball', 'walking',
'workplacework']
#create a table of activity impact on my mood
mood_inc=pd.DataFrame()
for col in cols:
mood_inc=mood_inc.append({'Activity':col,'Mood_bonus':(mm.pivot_table(index=col, values='mood_float', aggfunc='mean').mood_float[1]-mm.pivot_table(index=col, values='mood_float', aggfunc='mean').mood_float[0])*100},ignore_index=True)
# Merge mood with the number of its occurencies
# that's actually very ugly but I was very down doing that so I leave it like that
mood_occ=mm[cols].mean()*100
mood_df=pd.DataFrame(mood_occ)
mood_df=mood_df.reset_index()
mood_df.rename(columns={'0':'Mood_occ'},inplace=True)
mood_df['Mood_occ']=mood_df[0]
mood_inc_occ=pd.concat([mood_inc,mood_df], axis=1)
mood_inc_occ.drop('index', axis=1,inplace=True)
mood_inc_occ=mood_inc_occ[['Activity','Mood_bonus','Mood_occ']]
mood_inc_occ.sort_values(by='Mood_bonus',ascending=False,inplace=True)
mood_inc_occ=mood_inc_occ.round(1)
In [29]:
#mood_inc
#mood_occ
mood_inc_occ.head(6)
Out[29]:
Mood by activity¶
 |
 |
Do I need to walk to be happier ?¶
In [30]:
# What amount of steps in a day makes me happy?
mm_st.pivot_table(index='mood_float', values='steps', aggfunc='mean')
Out[30]:
In [31]:
mm_st.pivot_table(index='mood_float', values='steps', aggfunc='mean').plot(kind='bar',xlabel='Mood', ylabel='Steps',figsize=(12,8)).legend(['I like to walk']);
How much money spent makes me happy ?¶
In [32]:
# What amount of money spent in a day makes me happy?
mm.pivot_table(index='mood_float', values='amount_spent', aggfunc='mean')
Out[32]:
In [33]:
mm.pivot_table(index='mood_float', values='amount_spent', aggfunc='mean').plot(kind='bar',xlabel='Mood', ylabel='Money',figsize=(12,8)).legend(['Does anyone need a trendline here?']);
Predict¶
In [34]:
mm['weekday']=mm.date.dt.weekday
mm['month']=mm.date.dt.month
mm['day_of_month']=mm.date.dt.day
mm['weekend'] = np.where(mm['weekday']<5, 0, 1) #weekend or not
In [35]:
#create normalized dataframe
mmn=mm.copy()
#normalize 2 columns - float and amount
mmn['mood_float']=(mm.mood_float-mm.mood_float.mean())/mm.mood_float.std()
mmn['amount_spent']=(mm.amount_spent-mm.amount_spent.mean())/mm.amount_spent.std()
mmn.drop('date', axis=1, inplace=True)
Start of Eldiias crazy coding¶
In [36]:
y=mmn.mood_float
X=mmn.drop('mood_float',axis=1)
In [37]:
from sklearn.decomposition import PCA
pca=PCA(7)
pca.fit(X)
x_new=pca.transform(X)
In [38]:
pca.explained_variance_ratio_.sum()
Out[38]:
In [39]:
from statsmodels.api import OLS
from statsmodels.api import add_constant
In [40]:
from sklearn.tree import DecisionTreeClassifier
In [41]:
y
Out[41]:
In [42]:
kkk=DecisionTreeClassifier().fit(x_new,(mm.mood_float*2).astype(int))
In [43]:
plt.hist(kkk.predict(x_new))
plt.show()
(mm.mood_float*2).astype(int).hist()
Out[43]:
In [44]:
mm.mood_float.unique()
Out[44]:
In [45]:
mm.mood_float/5
Out[45]:
In [46]:
from sklearn.metrics import r2_score
In [47]:
r2_score((mm.mood_float*2).astype(int), kkk.predict(x_new))
Out[47]:
In [48]:
from sklearn.metrics import confusion_matrix
In [49]:
#that thing says that I have 100% SUPER PREDICTION
confusion_matrix((mm.mood_float*2).astype(int), kkk.predict(x_new))
Out[49]:
In [50]:
pca.components_
Out[50]:
- PCA
- test_train_split
- DecisionTreeClassifier
Confusion Matrix
DecisionTreeClassifier based on data without transformations
End of Eldiias crazy coding¶
In [51]:
# apparently, I can get a decision tree
# function to predict my mood based on parameters - must work with a group of parameters
# I can run Monte Carlo test on 1 million lines to get the activities
In [52]:
#recap
Learnings¶
- data collection is very important
Improvements¶
- how much money do I spend by walk
- how much money do I spend by pulse
- how much money do I spend by sleep
- how much money do I spend by mood activity
- how much hours do I sleep by spend
- how much hours do I sleep by walk
- how much hours do I sleep by mood
- how much stress do I stress by stress (pulse)
- sleep hours
- pulse data
- weather data as +
- income data
If I was to start from a scratch...¶
- get to Russia faster
Bonus - sleep¶
In [53]:
sleep=pd.read_csv('miband_data/SLEEP.csv')
sleep.date=pd.to_datetime(sleep.date)
sleep.drop_duplicates(subset=None, keep='first', inplace=True)
In [54]:
sleep['sleep_time'] = (sleep['deepSleepTime'] + sleep['shallowSleepTime'])/60
In [55]:
#data is actually limited by miband to 2019 and 2020, here I will limit it to the same limit as before
sleep=sleep.query('date<="2020-07-31"')
In [56]:
# no need for pivot table here but I copy the code from previous ones so it's easier
sleep_pivot=sleep.pivot_table(index='date', values='sleep_time', aggfunc='mean')
#join
mm_sl=mm.join(sleep_pivot, on='date')
#fillna with average - fillna doesn't work
mm_sl['sleep_time_right'] = np.where(mm_sl['sleep_time']==0, mm_sl['sleep_time'].median(),mm_sl['sleep_time'])
In [57]:
mm_sl.pivot_table(index='mood_float', values='sleep_time_right', aggfunc='mean').plot(kind='bar',xlabel='Mood', ylabel='Sleep',figsize=(12,8)).legend(['I do not care about sleeping']);
In [58]:
sleep.sleep_time.plot(kind='box')
Out[58]:
Bonus - pulse¶
In [59]:
pulse=pd.read_csv('miband_data/HEARTRATE_AUTO.csv')
#pulse.drop_duplicates(subset=None, keep='first', inplace=True) - no duplicates
pulse.date=pd.to_datetime(pulse.date)
pulse.time=pd.to_datetime(pulse.time)
pulse.sort_values(by='date',inplace=True)
In [60]:
pulse.pivot_table(index='date', values='heartRate', aggfunc='mean').plot(kind='box')
Out[60]:
In [61]:
# no need for pivot table here but I copy the code from previous ones so it's easier
pulse_pivot=pulse.pivot_table(index='date', values='heartRate', aggfunc='mean')
#join
mm_pu=mm.join(pulse_pivot, on='date')
# NO fillna with average - a lot of data is missing - DROPNA
mm_pu.dropna(inplace=True)
In [62]:
mm_pu.pivot_table(index='mood_float', values='heartRate', aggfunc='mean').plot(kind='bar',xlabel='Mood', ylabel='Pulse',figsize=(12,8)).set_ylim(71,81);
In [63]:
sns.lineplot(data=pulse, x="time", y="heartRate")
Out[63]:
In [64]:
import plotly
plotly.offline.init_notebook_mode()